<!DOCTYPE html>
<html xmlns="http://www.w3.org/1999/xhtml">
  <head>

    <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
    <meta content="Cask Data, Inc." name="author" />
<meta content="Copyright © 2014-2016 Cask Data, Inc." name="copyright" />


    <meta name="git_release" content="6.1.1">
    <meta name="git_hash" content="05fbac36f9f7aadeb44f5728cea35136dbc243e5">
    <meta name="git_timestamp" content="2020-02-09 08:22:47 +0800">
    <title>Iterative Data Processing with Apache Spark</title>

    <link rel="stylesheet" href="../_static/cdap-bootstrap.css" type="text/css" />
    <link rel="stylesheet" href="../_static/pygments.css" type="text/css" />
    <link rel="stylesheet" href="../_static/bootstrap-3.3.6/css/bootstrap.min.css" type="text/css" />
    <link rel="stylesheet" href="../_static/bootstrap-3.3.6/css/bootstrap-theme.min.css" type="text/css" />
    <link rel="stylesheet" href="../_static/css/bootstrap-sphinx.css" type="text/css" />
    <link rel="stylesheet" href="../_static/css/cdap-dynamicscrollspy-4.css" type="text/css" />
    <link rel="stylesheet" href="../_static/css/jquery.mCustomScrollbar.css" type="text/css" />
    <link rel="stylesheet" href="../_static/css/cdap-jquery.mCustomScrollbar.css" type="text/css" />
    <link rel="stylesheet" href="../_static/css/abixTreeList-2.css" type="text/css" />
    <link rel="stylesheet" href="../_static/cdap-bootstrap.css" type="text/css" />

    <script type="text/javascript">
      var DOCUMENTATION_OPTIONS = {
        URL_ROOT:    '',
        VERSION:     '6.1.1',
        COLLAPSE_INDEX: false,
        FILE_SUFFIX: '.html',
        HAS_SOURCE:  false
      };
    </script>
    <script type="text/javascript" src="../_static/jquery.js"></script>
    <script type="text/javascript" src="../_static/underscore.js"></script>
    <script type="text/javascript" src="../_static/doctools.js"></script>
    <script type="text/javascript" src="../_static/language_data.js"></script>

    <link rel="shortcut icon" href="../_static/favicon.ico"/>
    <link rel="index" title="Index" href="../genindex.html" />
    <link rel="search" title="Search" href="../search.html" />
    <link rel="top" title="Cask Data Application Platform 6.1.1 Documentation" href="../index.html" />
    <link rel="up" title="How-To Guides" href="index.html" />
    <link rel="next" title="Storing Timeseries Data" href="cdap-timeseries-guide.html" />
    <link rel="prev" title="Batch Data Processing with CDAP" href="cdap-mapreduce-guide.html" />
    <!-- block extrahead -->
    <meta charset='utf-8'>
    <meta http-equiv='X-UA-Compatible' content='IE=edge,chrome=1'>
    <meta name='viewport' content='width=device-width, initial-scale=1.0, maximum-scale=1'>
    <meta name="apple-mobile-web-app-capable" content="yes">
    <!-- block extrahead end -->

</head>
<body role="document">

<!-- block navbar -->
<div id="navbar" class="navbar navbar-inverse navbar-default navbar-fixed-top">
    <div class="container-fluid">
      <div class="row">
        <div class="navbar-header">
          <!-- .btn-navbar is used as the toggle for collapsed navbar content -->
          <a class="navbar-brand" href="../table-of-contents/../../index.html">
            <span><img alt="CDAP logo" src="../_static/cdap_logo.svg"/></span>
          </a>

          <button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".nav-collapse">
            <span class="icon-bar"></span>
            <span class="icon-bar"></span>
            <span class="icon-bar"></span>
          </button>

          <div class="pull-right">
            <div class="dropdown version-dropdown">
              <a href="#" class="dropdown-toggle" data-toggle="dropdown"
                role="button" aria-haspopup="true" aria-expanded="false">
                v 6.1.1 <span class="caret"></span>
              </a>
              <ul class="dropdown-menu">
                <li><a href="//docs.cdap.io/cdap/5.1.2/en/index.html">v 5.1.2</a></li>
                <li><a href="//docs.cdap.io/cdap/4.3.4/en/index.html">v 4.3.4</a></li>
              </ul>
            </div>
          </div>
          <form class="navbar-form navbar-right navbar-search" action="../search.html" method="get">
            <div class="form-group">
              <div class="navbar-search-image material-icons"></div>
              <input type="text" name="q" class="form-control" placeholder="  Search" />
            </div>
            <input type="hidden" name="check_keywords" value="yes" />
            <input type="hidden" name="area" value="default" />
          </form>

          <div class="collapse navbar-collapse nav-collapse navbar-right navbar-navigation">
            <ul class="nav navbar-nav"><li class="docsite-nav-tab-container"><a class="docsite-nav-tab-link " href="../table-of-contents/../../index.html">简介</a></li><li class="docsite-nav-tab-container"><a class="docsite-nav-tab-link current" href="../table-of-contents/../../guides.html">手册</a></li><li class="docsite-nav-tab-container"><a class="docsite-nav-tab-link " href="../table-of-contents/../../reference-manual/index.html">参考</a></li><li class="docsite-nav-tab-container"><a class="docsite-nav-tab-link " href="../table-of-contents/../../faqs/index.html">帮助</a></li>
            </ul>
          </div>

        </div>
      </div>
    </div>
  </div><!-- block navbar end -->
<!-- block main content -->
<div class="main-container container">
  <div class="row"><div class="col-md-2">
      <div id="sidebar" class="bs-sidenav scrollable-y-outside" role="complementary">
<!-- theme_manual: examples-manual -->
<!-- theme_manual_highlight: guides -->
<!-- sidebar_title_link: ../table-of-contents/../../guides.html -->

  <div role="note" aria-label="manuals links"><h3><a href="../table-of-contents/../../guides.html">Guides</a></h3>

    <ul class="this-page-menu">
      <li class="toctree-l1"><a href="../table-of-contents/../../user-guide/index.html" rel="nofollow">用户手册</a>
      </li>
      <li class="toctree-l1"><a href="../table-of-contents/../../developer-manual/index.html" rel="nofollow">开发手册</a>
      </li>
      <li class="toctree-l1"><a href="../table-of-contents/../../admin-manual/index.html" rel="nofollow">管理手册</a>
      </li>
      <li class="toctree-l1"><a href="../table-of-contents/../../integrations/index.html" rel="nofollow">集成手册</a>
      </li>
      <li class="toctree-l1"><b><a href="../table-of-contents/../../examples-manual/index.html" rel="nofollow">最佳实践</a></b>
      <nav class="pagenav">
      <ul class="current">
<li class="toctree-l1"><a class="reference internal" href="../index.html">Introduction</a></li>
<li class="toctree-l1 current"><a class="reference internal" href="index.html">How-To Guides</a><ul class="current">
<li class="toctree-l2"><a class="reference internal" href="cdap-bi-guide.html">Analyzing CDAP Data from BI Tools</a></li>
<li class="toctree-l2"><a class="reference internal" href="cdap-cube-guide.html">Data Analysis with OLAP Cube</a></li>
<li class="toctree-l2"><a class="reference internal" href="cdap-etl-guide.html">Creating Hydrator Applications using CDAP System Artifacts</a></li>
<li class="toctree-l2"><a class="reference internal" href="cdap-flow-guide.html">Real-Time Data Processing with a Flow</a></li>
<li class="toctree-l2"><a class="reference internal" href="cdap-flume-guide.html">Ingesting Data into CDAP using Apache Flume</a></li>
<li class="toctree-l2"><a class="reference internal" href="cdap-kafka-ingest-guide.html">Consuming Data from Kafka</a></li>
<li class="toctree-l2"><a class="reference internal" href="cdap-mapreduce-guide.html">Batch Data Processing with CDAP</a></li>
<li class="toctree-l2 current"><a class="current reference internal" href="#">Iterative Data Processing with Apache Spark</a></li>
<li class="toctree-l2"><a class="reference internal" href="cdap-timeseries-guide.html">Storing Timeseries Data</a></li>
<li class="toctree-l2"><a class="reference internal" href="cdap-twitter-ingest-guide.html">Consuming Twitter Data in Real Time</a></li>
<li class="toctree-l2"><a class="reference internal" href="cdap-workflow-guide.html">Batch Data Processing with CDAP using Workflow</a></li>
</ul>
</li>
</ul>
</nav>
      </li>
    </ul>
  </div></div>
    </div><div class="col-md-8 content" id="main-content">
    
  <div class="section" id="iterative-data-processing-with-apache-spark">
<span id="cdap-spark-guide"></span><h1>Iterative Data Processing with Apache Spark<a class="headerlink" href="#iterative-data-processing-with-apache-spark" title="Permalink to this headline">🔗</a></h1>
<blockquote class="pull-quote">
<div><strong>Source Code Repository:</strong> Source code (and other resources) for this guide are available at the
<a class="reference external" href="https://github.com/cdap-guides/cdap-spark-guide/tree/develop">CDAP Guides GitHub repository</a>.</div></blockquote>
<p><a class="reference external" href="https://spark.apache.org/">Apache Spark</a> is a very popular engine to
perform in-memory cluster computing for Apache Hadoop. In this guide,
you will learn how to run Apache Spark programs with CDAP.</p>
<div class="section" id="what-you-will-build">
<h2>What You Will Build<a class="headerlink" href="#what-you-will-build" title="Permalink to this headline">🔗</a></h2>
<p>You will build a
<a class="reference external" href="http://docs.cdap.io/cdap/current/en/developers-manual/building-blocks/applications.html">CDAP application</a>
that exposes a RESTful API to take in web pages’ backlinks information and
serve out the <a class="reference external" href="http://en.wikipedia.org/wiki/PageRank">PageRank</a> for the
known web pages. You will:</p>
<ul class="simple">
<li>Use a
<a class="reference external" href="http://docs.cdap.io/cdap/current/en/developers-manual/building-blocks/streams.html">Stream</a>
as the source of backlinks data;</li>
<li>Build a CDAP
<a class="reference external" href="http://docs.cdap.io/cdap/current/en/developers-manual/building-blocks/spark-jobs.html">Spark</a>
program that reads directly from the Stream and computes the PageRank of the web pages;</li>
<li>Use a
<a class="reference external" href="http://docs.cdap.io/cdap/current/en/developers-manual/building-blocks/datasets/index.html">Dataset</a>
to store the output of the Spark program; and</li>
<li>Build a
<a class="reference external" href="http://docs.cdap.io/cdap/current/en/developers-manual/building-blocks/services.html">Service</a>
to serve the PageRank computation results over HTTP.</li>
</ul>
</div>
<div class="section" id="what-you-will-need">
<h2>What You Will Need<a class="headerlink" href="#what-you-will-need" title="Permalink to this headline">🔗</a></h2>
<ul class="simple">
<li><a class="reference external" href="http://www.oracle.com/technetwork/java/javase/downloads/index.html">JDK 7 or 8</a></li>
<li><a class="reference external" href="http://maven.apache.org/">Apache Maven 3.1+</a></li>
<li><a class="reference external" href="http://docs.cdap.io/cdap/current/en/developers-manual/getting-started/standalone/index.html">CDAP SDK</a></li>
</ul>
</div>
<div class="section" id="lets-build-it">
<h2>Let’s Build It!<a class="headerlink" href="#lets-build-it" title="Permalink to this headline">🔗</a></h2>
<p>The following sections will guide you through building an application from scratch. If you
are interested in deploying and running the application right away, you can clone its
source code from this GitHub repository. In that case, feel free to skip the next two
sections and jump right to the
<a class="reference external" href="#build-and-run-application">Build and Run Application</a> section.</p>
<div class="section" id="application-design">
<h3>Application Design<a class="headerlink" href="#application-design" title="Permalink to this headline">🔗</a></h3>
<p>Backlinks data is sent to the <em>backlinkURLStream</em> over HTTP (e.g. by a web
crawler as it processes web pages). The PageRank for known pages is
computed periodically by a <em>PageRankProgram</em>. The program uses the
<em>backlinkURLStream</em> as an input and persists the results in the
<em>pageRanks</em> dataset.</p>
<p>The <em>PageRankService</em> then uses the <em>pageRanks</em> dataset to serve the
PageRank for a given URL over HTTP.</p>
<p>In this guide we assume that the backlinks data will be sent to a CDAP
application.</p>
<a class="reference internal image-reference" href="../_images/app-design6.png"><img alt="../_images/app-design6.png" class="align-center" src="../_images/app-design6.png" style="width: 8in;" /></a>
</div>
<div class="section" id="implementation">
<h3>Implementation<a class="headerlink" href="#implementation" title="Permalink to this headline">🔗</a></h3>
<p>The first step is to construct the application structure. We will use a
standard Maven project structure for all of the source code files:</p>
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="go">./pom.xml</span>
<span class="go">./src/main/java/co/cask/cdap/guides/PageRankApp.java</span>
<span class="go">./src/main/java/co/cask/cdap/guides/PageRankSpark.java</span>
<span class="go">./src/main/java/co/cask/cdap/guides/PageRankHandler.java</span>
<span class="go">./src/main/scala/co/cask/cdap/guides/PageRankProgram.scala</span>
</pre></div>
</div>
<p>The application is identified by the <code class="docutils literal notranslate"><span class="pre">PageRankApp</span></code> class. This class
extends <a class="reference external" href="http://docs.cdap.io/cdap/current/en/reference-manual/javadocs/co/cask/cdap/api/app/AbstractApplication.html">AbstractApplication</a>,
and overrides the <code class="docutils literal notranslate"><span class="pre">configure()</span></code> method to define all of the application components:</p>
<div class="highlight-java notranslate"><div class="highlight"><pre><span></span><span class="kd">public</span> <span class="kd">class</span> <span class="nc">PageRankApp</span> <span class="kd">extends</span> <span class="n">AbstractApplication</span> <span class="p">{</span>

  <span class="kd">public</span> <span class="kd">static</span> <span class="kd">final</span> <span class="n">String</span> <span class="n">PAGE_RANK_RANKS_SERVICE</span> <span class="o">=</span> <span class="s">&quot;PageRankService&quot;</span><span class="p">;</span>
  <span class="kd">public</span> <span class="kd">static</span> <span class="kd">final</span> <span class="n">String</span> <span class="n">PAGE_RANK_BACKLINK_STREAM</span> <span class="o">=</span> <span class="s">&quot;backlinkURLStream&quot;</span><span class="p">;</span>
  <span class="kd">public</span> <span class="kd">static</span> <span class="kd">final</span> <span class="n">String</span> <span class="n">PAGE_RANK_RANKS_DATASET</span> <span class="o">=</span> <span class="s">&quot;pageRanks&quot;</span><span class="p">;</span>

  <span class="nd">@Override</span>
  <span class="kd">public</span> <span class="kt">void</span> <span class="nf">configure</span><span class="p">()</span> <span class="p">{</span>
    <span class="n">addSpark</span><span class="p">(</span><span class="k">new</span> <span class="n">PageRankSpark</span><span class="p">());</span>
    <span class="n">addStream</span><span class="p">(</span><span class="k">new</span> <span class="n">Stream</span><span class="p">(</span><span class="n">PAGE_RANK_BACKLINK_STREAM</span><span class="p">));</span>
    <span class="n">addService</span><span class="p">(</span><span class="n">PAGE_RANK_RANKS_SERVICE</span><span class="p">,</span> <span class="k">new</span> <span class="n">PageRankHandler</span><span class="p">());</span>
    <span class="k">try</span> <span class="p">{</span>
      <span class="n">ObjectStores</span><span class="p">.</span><span class="na">createObjectStore</span><span class="p">(</span><span class="n">getConfigurer</span><span class="p">(),</span> <span class="n">PAGE_RANK_RANKS_DATASET</span><span class="p">,</span> <span class="n">Double</span><span class="p">.</span><span class="na">class</span><span class="p">);</span>
    <span class="p">}</span> <span class="k">catch</span> <span class="p">(</span><span class="n">UnsupportedTypeException</span> <span class="n">e</span><span class="p">)</span> <span class="p">{</span>
      <span class="k">throw</span> <span class="k">new</span> <span class="n">RuntimeException</span><span class="p">(</span><span class="s">&quot;Will never happen: all classes above are supported&quot;</span><span class="p">,</span> <span class="n">e</span><span class="p">);</span>
    <span class="p">}</span>
  <span class="p">}</span>
<span class="p">}</span>
</pre></div>
</div>
<p>In this example, we use a Stream to supply backlinks data;
the Spark program that computes the PageRank of the web pages reads directly from the Stream.
<code class="docutils literal notranslate"><span class="pre">backlinkURLStream</span></code> receives backlinks information in the form of two URLs separated by whitespace:</p>
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="go">http://example.com/page1 http://example.com/page10</span>
</pre></div>
</div>
<p>We’ll use Scala to write the Spark program (for an example of using Java, refer to the <a class="reference external" href="http://docs.cask.co/cdap/current/en/examples-manual/examples/spark-page-rank.html">CDAP SparkPageRank
example</a>).
You’ll need to add <code class="docutils literal notranslate"><span class="pre">scala</span></code> and <code class="docutils literal notranslate"><span class="pre">maven-scala-plugin</span></code> as dependencies in
your Maven <a class="reference external" href="https://github.com/cdap-guides/cdap-spark-guide/blob/develop/pom.xml">pom.xml</a>.</p>
<p>The code below configures Spark in CDAP. This class extends
<a class="reference external" href="http://docs.cdap.io/cdap/current/en/reference-manual/javadocs/co/cask/cdap/api/spark/AbstractSpark.html">AbstractSpark</a>
and overrides the <code class="docutils literal notranslate"><span class="pre">configure()</span></code> method to define all of the components. The
<code class="docutils literal notranslate"><span class="pre">setMainClassName</span></code> method sets the Spark Program class which CDAP will run:</p>
<div class="highlight-java notranslate"><div class="highlight"><pre><span></span><span class="kd">public</span> <span class="kd">class</span> <span class="nc">PageRankSpark</span> <span class="kd">extends</span> <span class="n">AbstractSpark</span> <span class="p">{</span>

  <span class="nd">@Override</span>
  <span class="kd">public</span> <span class="kt">void</span> <span class="nf">configure</span><span class="p">()</span> <span class="p">{</span>
    <span class="n">setName</span><span class="p">(</span><span class="n">PageRankSpark</span><span class="p">.</span><span class="na">class</span><span class="p">.</span><span class="na">getSimpleName</span><span class="p">());</span>
    <span class="n">setDescription</span><span class="p">(</span><span class="s">&quot;Spark program to compute PageRank&quot;</span><span class="p">);</span>
    <span class="n">setMainClass</span><span class="p">(</span><span class="n">PageRankProgram</span><span class="p">.</span><span class="na">class</span><span class="p">);</span>
  <span class="p">}</span>
<span class="p">}</span>
</pre></div>
</div>
<p>The <code class="docutils literal notranslate"><span class="pre">PageRankProgram</span></code> Spark program does the actual page rank
computation. This code is taken from the <a class="reference external" href="https://github.com/apache/spark/blob/master/examples/src/main/scala/org/apache/spark/examples/SparkPageRank.scala">Apache Spark’s PageRank example</a>;
the Spark program stores the computed PageRank in an ObjectStore
Dataset where the key is the URL and the value is the computed PageRank:</p>
<div class="highlight-java notranslate"><div class="highlight"><pre><span></span><span class="kd">class</span> <span class="nc">PageRankProgram</span> <span class="kd">extends</span> <span class="n">SparkMain</span> <span class="p">{</span>

  <span class="kd">private</span> <span class="kd">final</span> <span class="n">val</span> <span class="n">ITERATIONS_COUNT</span><span class="p">:</span> <span class="n">Int</span> <span class="o">=</span> <span class="mi">10</span>

  <span class="n">override</span> <span class="n">def</span> <span class="nf">run</span><span class="p">(</span><span class="n">implicit</span> <span class="n">sec</span><span class="p">:</span> <span class="n">SparkExecutionContext</span><span class="p">)</span> <span class="p">{</span>
    <span class="n">val</span> <span class="n">sc</span> <span class="o">=</span> <span class="k">new</span> <span class="n">SparkContext</span>
    <span class="n">val</span> <span class="n">lines</span><span class="p">:</span> <span class="n">RDD</span><span class="o">[</span><span class="n">String</span><span class="o">]</span> <span class="o">=</span> <span class="n">sc</span><span class="p">.</span><span class="na">fromStream</span><span class="p">(</span><span class="s">&quot;backlinkURLStream&quot;</span><span class="p">)</span>
    <span class="n">val</span> <span class="n">links</span> <span class="o">=</span> <span class="n">lines</span><span class="p">.</span><span class="na">map</span> <span class="p">{</span> <span class="n">s</span> <span class="o">=&gt;</span>
      <span class="n">val</span> <span class="n">parts</span> <span class="o">=</span> <span class="n">s</span><span class="p">.</span><span class="na">split</span><span class="p">(</span><span class="s">&quot;\\s+&quot;</span><span class="p">)</span>
      <span class="p">(</span><span class="n">parts</span><span class="p">(</span><span class="mi">0</span><span class="p">),</span> <span class="n">parts</span><span class="p">(</span><span class="mi">1</span><span class="p">))</span>
    <span class="p">}.</span><span class="na">distinct</span><span class="p">().</span><span class="na">groupByKey</span><span class="p">().</span><span class="na">cache</span><span class="p">()</span>

    <span class="kd">var</span> <span class="n">ranks</span> <span class="o">=</span> <span class="n">links</span><span class="p">.</span><span class="na">mapValues</span><span class="p">(</span><span class="n">v</span> <span class="o">=&gt;</span> <span class="mf">1.0</span><span class="p">)</span>

    <span class="c1">// Calculate the PageRanks</span>
    <span class="k">for</span> <span class="p">(</span><span class="n">i</span> <span class="o">&lt;-</span> <span class="mi">1</span> <span class="n">to</span> <span class="n">ITERATIONS_COUNT</span><span class="p">)</span> <span class="p">{</span>
      <span class="n">val</span> <span class="n">contribs</span> <span class="o">=</span> <span class="n">links</span><span class="p">.</span><span class="na">join</span><span class="p">(</span><span class="n">ranks</span><span class="p">).</span><span class="na">values</span><span class="p">.</span><span class="na">flatMap</span> <span class="p">{</span> <span class="k">case</span> <span class="p">(</span><span class="n">urls</span><span class="p">,</span> <span class="n">rank</span><span class="p">)</span> <span class="o">=&gt;</span>
        <span class="n">val</span> <span class="n">size</span> <span class="o">=</span> <span class="n">urls</span><span class="p">.</span><span class="na">size</span>
        <span class="n">urls</span><span class="p">.</span><span class="na">map</span><span class="p">(</span><span class="n">url</span> <span class="o">=&gt;</span> <span class="p">(</span><span class="n">url</span><span class="p">,</span> <span class="n">rank</span> <span class="o">/</span> <span class="n">size</span><span class="p">))</span>
      <span class="p">}</span>
      <span class="n">ranks</span> <span class="o">=</span> <span class="n">contribs</span><span class="p">.</span><span class="na">reduceByKey</span><span class="p">(</span><span class="n">_</span> <span class="o">+</span> <span class="n">_</span><span class="p">).</span><span class="na">mapValues</span><span class="p">(</span><span class="mf">0.15</span> <span class="o">+</span> <span class="mf">0.85</span> <span class="o">*</span> <span class="n">_</span><span class="p">)</span>
    <span class="p">}</span>

    <span class="n">ranks</span><span class="p">.</span><span class="na">map</span><span class="p">(</span><span class="n">x</span> <span class="o">=&gt;</span> <span class="p">(</span><span class="n">Bytes</span><span class="p">.</span><span class="na">toBytes</span><span class="p">(</span><span class="n">x</span><span class="p">.</span><span class="na">_1</span><span class="p">),</span> <span class="n">x</span><span class="p">.</span><span class="na">_2</span><span class="p">)).</span><span class="na">saveAsDataset</span><span class="p">(</span><span class="s">&quot;pageRanks&quot;</span><span class="p">)</span>
  <span class="p">}</span>
<span class="p">}</span>
</pre></div>
</div>
<p>To serve results out via HTTP, add a <code class="docutils literal notranslate"><span class="pre">PageRankHandler</span></code>, which
reads the PageRank for a given URL from the <code class="docutils literal notranslate"><span class="pre">pageRanks</span></code> dataset:</p>
<div class="highlight-java notranslate"><div class="highlight"><pre><span></span><span class="kd">public</span> <span class="kd">final</span> <span class="kd">class</span> <span class="nc">PageRankHandler</span> <span class="kd">extends</span> <span class="n">AbstractHttpServiceHandler</span> <span class="p">{</span>

  <span class="kd">private</span> <span class="kd">static</span> <span class="kd">final</span> <span class="n">Gson</span> <span class="n">GSON</span> <span class="o">=</span> <span class="k">new</span> <span class="n">Gson</span><span class="p">();</span>
  <span class="kd">public</span> <span class="kd">static</span> <span class="kd">final</span> <span class="n">String</span> <span class="n">URL_KEY</span> <span class="o">=</span> <span class="s">&quot;url&quot;</span><span class="p">;</span>
  <span class="kd">public</span> <span class="kd">static</span> <span class="kd">final</span> <span class="n">String</span> <span class="n">PAGE_RANKS_RANK_HANDLER</span> <span class="o">=</span> <span class="s">&quot;pagerank&quot;</span><span class="p">;</span>

  <span class="nd">@UseDataSet</span><span class="p">(</span><span class="n">PageRankApp</span><span class="p">.</span><span class="na">PAGE_RANK_RANKS_DATASET</span><span class="p">)</span>
  <span class="kd">private</span> <span class="n">ObjectStore</span><span class="o">&lt;</span><span class="n">Double</span><span class="o">&gt;</span> <span class="n">ranks</span><span class="p">;</span>

  <span class="nd">@Path</span><span class="p">(</span><span class="n">PAGE_RANKS_RANK_HANDLER</span><span class="p">)</span>
  <span class="nd">@POST</span>
  <span class="kd">public</span> <span class="kt">void</span> <span class="nf">getRank</span><span class="p">(</span><span class="n">HttpServiceRequest</span> <span class="n">request</span><span class="p">,</span> <span class="n">HttpServiceResponder</span> <span class="n">responder</span><span class="p">)</span> <span class="p">{</span>
    <span class="n">String</span> <span class="n">url</span> <span class="o">=</span> <span class="n">GSON</span><span class="p">.</span><span class="na">fromJson</span><span class="p">(</span><span class="n">Charsets</span><span class="p">.</span><span class="na">UTF_8</span><span class="p">.</span><span class="na">decode</span><span class="p">(</span><span class="n">request</span><span class="p">.</span><span class="na">getContent</span><span class="p">()).</span><span class="na">toString</span><span class="p">(),</span>
                               <span class="n">JsonObject</span><span class="p">.</span><span class="na">class</span><span class="p">).</span><span class="na">get</span><span class="p">(</span><span class="n">URL_KEY</span><span class="p">).</span><span class="na">getAsString</span><span class="p">();</span>
    <span class="k">if</span> <span class="p">(</span><span class="n">url</span> <span class="o">==</span> <span class="kc">null</span><span class="p">)</span> <span class="p">{</span>
      <span class="n">responder</span><span class="p">.</span><span class="na">sendError</span><span class="p">(</span><span class="n">HttpURLConnection</span><span class="p">.</span><span class="na">HTTP_BAD_REQUEST</span><span class="p">,</span> <span class="s">&quot;The url must be specified with url as key in JSON.&quot;</span><span class="p">);</span>
      <span class="k">return</span><span class="p">;</span>
    <span class="p">}</span>

    <span class="c1">// Get the rank from the ranks dataset</span>
    <span class="n">Double</span> <span class="n">rank</span> <span class="o">=</span> <span class="n">ranks</span><span class="p">.</span><span class="na">read</span><span class="p">(</span><span class="n">url</span><span class="p">.</span><span class="na">getBytes</span><span class="p">(</span><span class="n">Charsets</span><span class="p">.</span><span class="na">UTF_8</span><span class="p">));</span>
    <span class="k">if</span> <span class="p">(</span><span class="n">rank</span> <span class="o">==</span> <span class="kc">null</span><span class="p">)</span> <span class="p">{</span>
      <span class="n">responder</span><span class="p">.</span><span class="na">sendError</span><span class="p">(</span><span class="n">HttpURLConnection</span><span class="p">.</span><span class="na">HTTP_NO_CONTENT</span><span class="p">,</span> <span class="n">String</span><span class="p">.</span><span class="na">format</span><span class="p">(</span><span class="s">&quot;No rank found of %s&quot;</span><span class="p">,</span> <span class="n">url</span><span class="p">));</span>
    <span class="p">}</span> <span class="k">else</span> <span class="p">{</span>
      <span class="n">responder</span><span class="p">.</span><span class="na">sendString</span><span class="p">(</span><span class="n">rank</span><span class="p">.</span><span class="na">toString</span><span class="p">());</span>
    <span class="p">}</span>
  <span class="p">}</span>
<span class="p">}</span>
</pre></div>
</div>
</div>
</div>
<div class="section" id="build-and-run-application">
<h2>Build and Run Application<a class="headerlink" href="#build-and-run-application" title="Permalink to this headline">🔗</a></h2>
<p>The <code class="docutils literal notranslate"><span class="pre">PageRankApp</span></code> application can be built and packaged using the Apache Maven command:</p>
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="gp">$</span> mvn clean package
</pre></div>
</div>
<p>Note that the remaining commands assume that the <code class="docutils literal notranslate"><span class="pre">cdap-cli.sh</span></code> script is
available on your PATH. If this is not the case, please add it:</p>
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="gp">$</span> <span class="nb">export</span> <span class="nv">PATH</span><span class="o">=</span><span class="nv">$PATH</span>:&lt;CDAP home&gt;/bin
</pre></div>
</div>
<p>If you haven’t already started a standalone CDAP installation, start it with the command:</p>
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="gp">$</span> cdap.sh start
</pre></div>
</div>
<p>You can then deploy the application to a standalone CDAP installation:</p>
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="gp">$</span> cdap-cli.sh load artifact target/cdap-spark-guide-&lt;version&gt;.jar
<span class="gp">$</span> cdap-cli.sh create app PageRankApp cdap-spark-guide &lt;version&gt; user
</pre></div>
</div>
<p>Start the Service:</p>
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="gp">$</span> cdap-cli.sh start service PageRankApp.PageRankService
</pre></div>
</div>
<p>Send some Data to the Stream:</p>
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="gp">$</span> cdap-cli.sh send stream backlinkURLStream <span class="se">\&#39;</span>http://example.com/page1 http://example.com/page1<span class="se">\&#39;</span>
<span class="gp">$</span> cdap-cli.sh send stream backlinkURLStream <span class="se">\&#39;</span>http://example.com/page1 http://example.com/page10<span class="se">\&#39;</span>
<span class="gp">$</span> cdap-cli.sh send stream backlinkURLStream <span class="se">\&#39;</span>http://example.com/page10 http://example.com/page10<span class="se">\&#39;</span>
<span class="gp">$</span> cdap-cli.sh send stream backlinkURLStream <span class="se">\&#39;</span>http://example.com/page10 http://example.com/page100<span class="se">\&#39;</span>
<span class="gp">$</span> cdap-cli.sh send stream backlinkURLStream <span class="se">\&#39;</span>http://example.com/page100 http://example.com/page100<span class="se">\&#39;</span>
</pre></div>
</div>
<p>Run the Spark Program:</p>
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="gp">$</span> cdap-cli.sh start spark PageRankApp.PageRankSpark
</pre></div>
</div>
<p>The Spark Program can take time to complete. You can check the status
for completion using:</p>
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="gp">$</span> cdap-cli.sh get spark status PageRankApp.PageRankSpark
</pre></div>
</div>
<p>Query for the PageRank results:</p>
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="gp">$</span> cdap-cli.sh call service PageRankApp.PageRankService POST <span class="s1">&#39;pagerank&#39;</span> body <span class="s1">&#39;{&quot;url&quot;:&quot;http://example.com/page1&quot;}&#39;</span>
</pre></div>
</div>
<p>Example output:</p>
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="go">+====================================================================================+</span>
<span class="go">| status | headers                                  | body size | body               |</span>
<span class="go">+====================================================================================+</span>
<span class="go">| 200    | Content-Length : 18                      | 18        | 0.2610116705534049 |</span>
<span class="go">|        | Content-Type : text/plain; charset=utf-8 |           |                    |</span>
<span class="go">|        | Connection : keep-alive                  |           |                    |</span>
<span class="go">+====================================================================================+</span>
</pre></div>
</div>
<p>Congratulations! You have now learned how to incorporate Spark programs
into your CDAP applications. Please continue to experiment and extend
this sample application.</p>
</div>
<div class="section" id="share-and-discuss">
<h2>Share and Discuss!<a class="headerlink" href="#share-and-discuss" title="Permalink to this headline">🔗</a></h2>
<p>Have a question? Discuss at the <a class="reference external" href="https://groups.google.com/forum/#!forum/cdap-user">CDAP User Mailing List</a>.</p>
</div>
<div class="section" id="license">
<h2>License<a class="headerlink" href="#license" title="Permalink to this headline">🔗</a></h2>
<p>Copyright © 2014-2016 Cask Data, Inc.</p>
<p>Licensed under the Apache License, Version 2.0 (the “License”); you may
not use this file except in compliance with the License. You may obtain
a copy of the License at</p>
<p><a class="reference external" href="http://www.apache.org/licenses/LICENSE-2.0">http://www.apache.org/licenses/LICENSE-2.0</a></p>
<p>Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an “AS IS” BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.</p>
</div>
</div>

</div>
    <div class="col-md-2">
      <div id="right-sidebar" class="bs-sidenav scrollable-y" role="complementary">
        <div id="localtoc-scrollspy">
        </div>
      </div>
    </div></div>
</div>
<!-- block main content end -->
<!-- block footer -->
<footer class="footer">
      <div class="container">
        <div class="row">
          <div class="col-md-2 footer-left"><a title="Batch Data Processing with CDAP" href="cdap-mapreduce-guide.html" />Previous</a></div>
          <div class="col-md-8 footer-center"><a class="footer-tab-link" href="../table-of-contents/../../reference-manual/licenses/index.html">Copyright</a> &copy; 2014-2020 Cask Data, Inc.&bull; <a class="footer-tab-link" href="//docs.cask.co/cdap/6.1.1/cdap-docs-6.1.1-web.zip" rel="nofollow">Download</a> an archive or
<a class="footer-tab-link" href="//docs.cask.co/cdap">switch the version</a> of the documentation
          </div>
          <div class="col-md-2 footer-right"><a title="Storing Timeseries Data" href="cdap-timeseries-guide.html" />Next</a></div>
        </div>
      </div>
    </footer>
<!-- block footer end -->
<script type="text/javascript" src="../_static/bootstrap-3.3.6/js/bootstrap.min.js"></script><script type="text/javascript" src="../_static/js/bootstrap-sphinx.js"></script><script type="text/javascript" src="../_static/js/abixTreeList-2.js"></script><script type="text/javascript" src="../_static/js/cdap-dynamicscrollspy-4.js"></script><script type="text/javascript" src="../_static/js/cdap-version-menu.js"></script><script type="text/javascript" src="../_static/js/copy-to-clipboard.js"></script><script type="text/javascript" src="../_static/js/jquery.mousewheel.min.js"></script><script type="text/javascript" src="../_static/js/jquery.mCustomScrollbar.js"></script><script type="text/javascript" src="../_static/js/js.cookie.js"></script><script type="text/javascript" src="../_static/js/tabbed-parsed-literal-0.2.js"></script><script type="text/javascript" src="../_static/js/cdap-onload-javascript.js"></script><script type="text/javascript" src="../_static/js/cdap-version-menu.js"></script>
    <script src="https://cdap.gitee.io/docs/cdap/json-versions.js"/></script>
  </body>
</html>